{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\" style=\"margin-top: 1em;\"><ul class=\"toc-item\"></ul></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CNN for MNIST with TensorFlow and Keras <a class=\"tocSkip\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NumPy:1.13.1\n",
      "TensorFlow:1.4.1\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(123)\n",
    "print(\"NumPy:{}\".format(np.__version__))\n",
    "\n",
    "import tensorflow as tf\n",
    "tf.set_random_seed(123)\n",
    "print(\"TensorFlow:{}\".format(tf.__version__))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASETSLIB_HOME = '../datasetslib'\n",
    "import sys\n",
    "if not DATASETSLIB_HOME in sys.path:\n",
    "    sys.path.append(DATASETSLIB_HOME)\n",
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "import datasetslib\n",
    "\n",
    "datasetslib.datasets_root = os.path.join(os.path.expanduser('~'),'datasets')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get the MNIST data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /home/armando/datasets/mnist/train-images-idx3-ubyte.gz\n",
      "Extracting /home/armando/datasets/mnist/train-labels-idx1-ubyte.gz\n",
      "Extracting /home/armando/datasets/mnist/t10k-images-idx3-ubyte.gz\n",
      "Extracting /home/armando/datasets/mnist/t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(os.path.join(datasetslib.datasets_root,\n",
    "                                               'mnist'), \n",
    "                                  one_hot=True)\n",
    "\n",
    "X_train = mnist.train.images\n",
    "X_test = mnist.test.images\n",
    "Y_train = mnist.train.labels\n",
    "Y_test = mnist.test.labels\n",
    "n_classes = 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocess for RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X_train.reshape(-1,28,28)\n",
    "X_test = X_test.reshape(-1,28,28)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RNN With Keras for MNIST Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation\n",
    "from keras.layers.recurrent import SimpleRNN\n",
    "from keras.optimizers import RMSprop\n",
    "from keras.optimizers import SGD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "keras.backend.clear_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "simple_rnn_1 (SimpleRNN)     (None, 16)                720       \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 10)                170       \n",
      "_________________________________________________________________\n",
      "activation_1 (Activation)    (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 890\n",
      "Trainable params: 890\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/20\n",
      "55000/55000 [==============================] - 5s 84us/step - loss: 1.3061 - acc: 0.5084\n",
      "Epoch 2/20\n",
      "55000/55000 [==============================] - 5s 88us/step - loss: 1.0005 - acc: 0.6136\n",
      "Epoch 3/20\n",
      "55000/55000 [==============================] - 4s 77us/step - loss: 0.9252 - acc: 0.6544\n",
      "Epoch 4/20\n",
      "55000/55000 [==============================] - 5s 84us/step - loss: 0.8111 - acc: 0.7121\n",
      "Epoch 5/20\n",
      "55000/55000 [==============================] - 4s 78us/step - loss: 0.7383 - acc: 0.7450\n",
      "Epoch 6/20\n",
      "55000/55000 [==============================] - 4s 82us/step - loss: 0.6968 - acc: 0.7631\n",
      "Epoch 7/20\n",
      "55000/55000 [==============================] - 5s 84us/step - loss: 0.6685 - acc: 0.7725\n",
      "Epoch 8/20\n",
      "55000/55000 [==============================] - 4s 75us/step - loss: 0.6553 - acc: 0.7765\n",
      "Epoch 9/20\n",
      "55000/55000 [==============================] - 4s 77us/step - loss: 0.6455 - acc: 0.7851\n",
      "Epoch 10/20\n",
      "55000/55000 [==============================] - 5s 82us/step - loss: 0.6378 - acc: 0.7920\n",
      "Epoch 11/20\n",
      "55000/55000 [==============================] - 5s 84us/step - loss: 0.6313 - acc: 0.7983\n",
      "Epoch 12/20\n",
      "55000/55000 [==============================] - 4s 76us/step - loss: 0.6381 - acc: 0.7997\n",
      "Epoch 13/20\n",
      "55000/55000 [==============================] - 4s 81us/step - loss: 0.6193 - acc: 0.8049\n",
      "Epoch 14/20\n",
      "55000/55000 [==============================] - 4s 81us/step - loss: 0.6208 - acc: 0.8097\n",
      "Epoch 15/20\n",
      "55000/55000 [==============================] - 4s 79us/step - loss: 0.6021 - acc: 0.8132\n",
      "Epoch 16/20\n",
      "55000/55000 [==============================] - 4s 76us/step - loss: 0.6231 - acc: 0.8117\n",
      "Epoch 17/20\n",
      "55000/55000 [==============================] - 4s 74us/step - loss: 0.6085 - acc: 0.8164\n",
      "Epoch 18/20\n",
      "55000/55000 [==============================] - 5s 82us/step - loss: 0.6161 - acc: 0.8141\n",
      "Epoch 19/20\n",
      "55000/55000 [==============================] - 4s 81us/step - loss: 0.6146 - acc: 0.8146\n",
      "Epoch 20/20\n",
      "55000/55000 [==============================] - 4s 78us/step - loss: 0.5835 - acc: 0.8286\n",
      "10000/10000 [==============================] - 1s 119us/step\n",
      "\n",
      "Test loss: 0.520945608187\n",
      "Test accuracy: 0.8379\n"
     ]
    }
   ],
   "source": [
    "# create and fit the SimpleRNN model\n",
    "model = Sequential()\n",
    "model.add(SimpleRNN(units=16, activation='relu', input_shape=(28,28)))\n",
    "model.add(Dense(n_classes))\n",
    "model.add(Activation('softmax'))\n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=RMSprop(lr=0.01),\n",
    "              metrics=['accuracy'])\n",
    "model.summary()\n",
    "\n",
    "model.fit(X_train, Y_train,\n",
    "                    batch_size=100,\n",
    "                    epochs=20)\n",
    "\n",
    "score = model.evaluate(X_test, Y_test)\n",
    "print('\\nTest loss:', score[0])\n",
    "print('Test accuracy:', score[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "toc_cell": true,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
